{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.10 64-bit ('anaconda3': virtualenv)"
  },
  "interpreter": {
   "hash": "7b4c34fa5edc2b5c200e84280da452af41185f443fff3e767a73b82cf30c2550"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from pathlib import Path\n",
    "from common.configs.path import paths\n",
    "from common.configs.stopwords import punc, stopwords\n",
    "from common.configs.tools import label_map\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import PCA\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "source": [
    "# build tfidf"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(paths['train_data'])\n",
    "test_df = pd.read_csv(paths['test_data'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_ = pd.concat([train_df['text'], test_df['text']]).apply(lambda e: re.sub('|'.join(punc+stopwords), '', e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(20013, 3449)"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(ngram_range=(1, 1))\n",
    "tfidf_result = vectorizer.fit_transform(text_.tolist())\n",
    "tfidf_result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=1000, svd_solver='full')\n",
    "pca.fit(tfidf_result.toarray())\n",
    "tfidf_pca = pca.transform(tfidf_result.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(r'./tfidf_1gram_pca1000.npy', np.array(tfidf_pca))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(20013, 3000)"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "tfidf_result2g.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer2g = TfidfVectorizer(ngram_range=(2, 2), max_features=10000)\n",
    "tfidf_result2g = vectorizer2g.fit_transform(text_.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca2g = PCA(n_components=1000, svd_solver='full')\n",
    "tfidf_pca2g = pca2g.fit_transform(tfidf_result2g.toarray())\n",
    "# pca.transform(tfidf_result2g.toarray())\n",
    "np.save(r'./tfidf_2gram_pca1000.npy', np.array(tfidf_pca2g))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer3g = TfidfVectorizer(ngram_range=(3, 3), max_features=10000)\n",
    "tfidf_result3g = vectorizer2g.fit_transform(text_.tolist())\n",
    "pca3g = PCA(n_components=1000, svd_solver='full')\n",
    "tfidf_pca3g = pca3g.fit_transform(tfidf_result3g.toarray())\n",
    "\n",
    "np.save(r'./tfidf_3gram_pca1000.npy', np.array(tfidf_pca3g))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(20013, 3000)"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "con_tfidf = np.concatenate((tfidf_pca2g, tfidf_pca2g, tfidf_pca3g), axis=1)\n",
    "con_tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(r'./tfidf_con_3000.npy', np.array(con_tfidf))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = con_tfidf[:14009]\n",
    "test_data = con_tfidf[14009:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train_df.label.apply(lambda e: label_map[e]).values"
   ]
  },
  {
   "source": [
    "# build model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import CategoricalNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "import pickle\n",
    "\n",
    "def save_model(model, path):\n",
    "    with open(path, 'wb') as f:\n",
    "        pickle.dump(model, f)    \n",
    "\n",
    "def load_model(path):\n",
    "    with open(path, 'rb') as f:\n",
    "        model = pickle.load(f)\n",
    "    return model\n",
    "\n",
    "def load_npy(path):\n",
    "    return np.load(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "con_tfidf = load_npy(paths['output'] / 'tfidf/tfidf_con_3000.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc = SVC(class_weight='balanced')\n",
    "rf = RandomForestClassifier(max_depth=5, \n",
    "                            random_state=31, class_weight='balanced',\n",
    "                            n_jobs=-1)\n",
    "cnb = CategoricalNB()\n",
    "lr = LogisticRegression(random_state=0,\n",
    "                        multi_class='ovr', n_jobs=-1)\n",
    "knn = KNeighborsClassifier(n_neighbors=10, n_jobs=-1)\n",
    "lgbm = lgb.LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
    "                         importance_type='split', learning_rate=0.01, max_depth=-1,\n",
    "                         min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
    "                         n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
    "                         random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
    "                         subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-49-ffec4e219c78>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m                                          ('lgb', lgbm)],\n\u001b[1;32m     16\u001b[0m                              n_jobs=-1, verbose=True)\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0meclf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meclf1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meclf1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf1_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maverage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'macro'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/ensemble/_voting.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0mtransformed_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mle_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransformed_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/ensemble/_voting.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m     72\u001b[0m                              % (len(self.weights), len(self.estimators)))\n\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m         self.estimators_ = Parallel(n_jobs=self.n_jobs)(\n\u001b[0m\u001b[1;32m     75\u001b[0m                 delayed(_fit_single_estimator)(\n\u001b[1;32m     76\u001b[0m                         \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    964\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_managed_backend\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 966\u001b[0;31m             \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initialize_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    967\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m             \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_effective_n_jobs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_initialize_backend\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    731\u001b[0m         \u001b[0;34m\"\"\"Build a process or thread pool and return the number of workers\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    732\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 733\u001b[0;31m             n_jobs = self._backend.configure(n_jobs=self.n_jobs, parallel=self,\n\u001b[0m\u001b[1;32m    734\u001b[0m                                              **self._backend_args)\n\u001b[1;32m    735\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msupports_timeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mconfigure\u001b[0;34m(self, n_jobs, parallel, prefer, require, idle_worker_timeout, **memmappingexecutor_args)\u001b[0m\n\u001b[1;32m    492\u001b[0m                 SequentialBackend(nesting_level=self.nesting_level))\n\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 494\u001b[0;31m         self._workers = get_memmapping_executor(\n\u001b[0m\u001b[1;32m    495\u001b[0m             \u001b[0mn_jobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0midle_worker_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    496\u001b[0m             \u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_prepare_worker_env\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/executor.py\u001b[0m in \u001b[0;36mget_memmapping_executor\u001b[0;34m(n_jobs, **kwargs)\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_memmapping_executor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mMemmappingExecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_memmapping_executor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/executor.py\u001b[0m in \u001b[0;36mget_memmapping_executor\u001b[0;34m(cls, n_jobs, timeout, initializer, initargs, env, temp_folder, context_id, **backend_args)\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0mtemp_folder_resolver\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmanager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresolve_temp_folder_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             **backend_args)\n\u001b[0;32m---> 52\u001b[0;31m         _executor, executor_is_reused = super().get_reusable_executor(\n\u001b[0m\u001b[1;32m     53\u001b[0m             \u001b[0mn_jobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjob_reducers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjob_reducers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult_reducers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresult_reducers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/externals/loky/reusable_executor.py\u001b[0m in \u001b[0;36mget_reusable_executor\u001b[0;34m(cls, max_workers, context, timeout, kill_workers, reuse, job_reducers, result_reducers, initializer, initargs, env)\u001b[0m\n\u001b[1;32m    158\u001b[0m                         \u001b[0;34m\"previous instance cannot be reused ({}).\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m                         .format(max_workers, reason))\n\u001b[0;32m--> 160\u001b[0;31m                     \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkill_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkill_workers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m                     \u001b[0m_executor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecutor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_executor_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m                     \u001b[0;31m# Recursive call to build a new instance\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/externals/loky/process_executor.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, kill_workers)\u001b[0m\n\u001b[1;32m   1169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1170\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mexecutor_manager_thread\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1171\u001b[0;31m             \u001b[0mexecutor_manager_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1173\u001b[0m         \u001b[0;31m# To reduce the risk of opening too many files, remove references to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1009\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1010\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1011\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1012\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1025\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1026\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1027\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1028\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1029\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "kf = KFold(n_splits=5)\n",
    "\n",
    "X = train_data\n",
    "\n",
    "for k, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "    # print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "    eclf1 = VotingClassifier(estimators=[('svc', svc),\n",
    "                                         ('rf', rf), \n",
    "                                         ('cnb', cnb), \n",
    "                                         ('lr', lr), \n",
    "                                         ('knn', knn),\n",
    "                                         ('lgb', lgbm)],\n",
    "                             n_jobs=-1, verbose=True)\n",
    "    eclf1 = eclf1.fit(X_train, y_train)\n",
    "    y_pred = eclf1.predict(X_test)\n",
    "    f1 = f1_score(y_test, y_pred, average='macro')\n",
    "    print('[{}] f1: {}'.format(k, f1))\n",
    "    save_model(eclf1, './voting_model_{}.pkl'.format(k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec.load('./common/output/c2v_model/w2v_1gram.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([-0.7255346 , -0.21972777,  1.2817037 , -1.0416013 , -2.1137164 ,\n",
       "       -0.83243835,  1.8416246 ,  0.48093927, -0.3306551 ,  0.7394607 ,\n",
       "       -0.42676026,  0.5589183 ,  0.5622761 , -0.03153883, -0.02210858,\n",
       "        1.2483007 ,  0.55192643,  1.2232726 , -1.3846047 , -0.45819935,\n",
       "       -1.6222272 ,  0.10531961, -1.5823069 ,  1.7002444 ,  1.1572521 ,\n",
       "       -1.1097908 ,  0.26619673, -0.48973194, -0.11755111, -0.69259316,\n",
       "        1.3996177 ,  0.43225813,  0.9389288 ,  0.4411196 ,  0.33248067,\n",
       "       -0.853198  ,  0.9133305 , -1.2339743 ,  1.1098661 , -1.5233839 ,\n",
       "       -0.29421762,  0.74252725,  2.0084496 ,  0.3635195 ,  0.23449023,\n",
       "       -1.0956067 , -0.61529547,  0.91753954,  1.0154477 ,  1.4524413 ,\n",
       "        0.65956694,  2.059024  ,  0.6947612 ,  0.21294437,  0.08979909,\n",
       "        0.18295743,  2.2541451 ,  1.005962  ,  0.3108316 ,  0.6484509 ,\n",
       "        2.9407086 ,  0.08892468,  0.33987194,  0.25834966,  0.5042563 ,\n",
       "       -0.45151109, -1.1901618 ,  0.1115315 , -1.3491971 , -0.27883562,\n",
       "       -0.75040096, -1.1202681 ,  1.2646238 , -0.8417917 ,  0.861116  ,\n",
       "        0.17806622,  0.56357586,  0.9921731 , -0.01341907, -0.3675711 ,\n",
       "        0.16596332,  0.29073092,  0.21280523,  1.0857816 , -0.97105634,\n",
       "        0.06221263, -0.23591174, -0.1423247 ,  0.9317043 , -0.9420283 ,\n",
       "       -1.510192  , -0.9720601 , -0.11551648, -0.49772656, -0.04426593,\n",
       "       -0.5389989 ,  0.918931  , -0.29738805,  1.845671  , -1.3260479 ],\n",
       "      dtype=float32)"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "model.wv['26336']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}